# -*- coding: utf-8 -*-
import csv
import scrapy
from lxml import html as htm

# 模拟浏览器
USER_AGENT = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"

class RichSpider(scrapy.Spider):
    name = 'rich'

    def start_requests(self):
        #构造爬取地址列表
        urls = [
            "https://www.forbeschina.com/lists/1131",
            "https://www.forbeschina.com/lists/1162",
            "https://www.forbeschina.com/lists/1728"
        ]

        #写入成csv表格标题
        data = [['year','rank', 'name', 'money', 'company', 'age', 'city', 'industry']]

        #打开文件句柄
        file = open('rich.csv','a+',encoding ='utf-8', newline="")

        #csv操作对戏
        csv_writer = csv.writer(file)

        #写入csv表头
        csv_writer.writerows(data)

        #关闭文件流
        file.close()

        for url in urls:
            yield scrapy.Request(url=url,headers={"User-Agent": USER_AGENT},callback=self.parse)

    #解析请求结果
    def parse(self, response):
        year = 2017

        if "1131" in response.request.url:
            year = 2017
        if "1162" in response.request.url:
            year = 2018
        if "1728" in response.request.url:
            year = 2019

        html = htm.fromstring(response.body)

        trList = html.xpath('//table[@id="data-view"]/tbody/tr')

        total = []

        for i in trList:
            # 排名
            rank = i.xpath('./td[1]/text()')[0].replace(" ","")

            # 姓名
            name = i.xpath('./td[2]/text()')[0].replace(" ","")
            if len(name.split("/"))>0:
                name = name.split("/")[0]
            else:
                pass

            # 财富
            money = i.xpath('./td[3]/text()')[0].replace(" ","")

            if year == 2017:
                #年龄
                age = i.xpath('./td[4]/text()')[0].replace(" ","")

                # 公司
                company = ""

                # 所在城市
                city = i.xpath('./td[6]/text()')[0].replace(" ", "")

                # 行业
                industry = i.xpath('./td[5]/text()')[0].replace(" ", "")

            elif year == 2018:
                #年龄
                age = ""

                # 公司
                company = i.xpath('./td[4]/text()')[0].replace(" ", "")

                # 所在城市
                city = i.xpath('./td[5]/text()')[0].replace(" ", "")

                # 行业
                industry = i.xpath('./td[6]/text()')[0].replace(" ", "")

            else:
                # 公司
                company = i.xpath('./td[4]/text()')[0].replace(" ", "")

                # 年龄
                age = i.xpath('./td[5]/text()')[0].replace(" ", "")

                # 所在城市
                city = i.xpath('./td[6]/text()')[0].replace(" ", "")

                # 行业
                industry = i.xpath('./td[7]/text()')[0].replace(" ", "")

            print([year,rank,name,money,company,age,city,industry])

            try:
                age = int(age)
            except:
                age = ""

            total.append([year,rank,name,money,company,age,city,industry])

        file = open('rich.csv','a+',encoding ='utf-8', newline="")

        csv_writer = csv.writer(file)

        csv_writer.writerows(total)

        file.close()
